/*
 * 文件名：Login.java.
 * 版权：Copyright 2014-2017 上海酷方网络科技有限公司 All Rights Reserved.
 * 描述：
 * 修改人: Kevin
 * 修改时间：2017年4月11日
 * 修改单号：
 * 修改内容：
 */

package com.skf.mokao.sina;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.httpclient.Cookie;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.http.cookie.CookieSpec;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.DomSerializer;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XPatherException;
import org.jsoup.Jsoup;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

/**
 * 
 * 
 * @author Kevin
 * @version 2017年4月11日
 * @see
 * @since
 *
 */

public class Login {

	// public static Cookie[] getCookies() {
	// Cookie[] cookies =null;
	// HttpClient client;
	// PostMethod post = null;
	// String url = "https://passport.weibo.cn/sso/login";
	// post = new PostMethod(url);
	// post.setRequestHeader("User-Agent","Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.77 Safari/537.1");
	// post.setRequestHeader("Referer", "http://weibo.com/");
	// post.setRequestHeader("Content-Type",
	// "application/x-www-form-urlencoded");
	// post.setParameter("username", "gusean@live.cn");
	// post.setParameter("password", "13720329391");
	// post.setParameter("savestate", "1");
	// post.setParameter("entry", "mweibo");
	// post.setParameter("mainpageflag", "1");
	// post.setParameter("ec", "0");
	//
	// URL url2 = null;
	// try {
	// url2 = new URL(url);
	// } catch (MalformedURLException e1) {
	// // TODO Auto-generated catch block
	// e1.printStackTrace();
	// }
	// client = new HttpClient();
	// client.startSession(url2);
	// try {
	// client.executeMethod(post);
	// } catch (IOException | HttpException e) {
	// e.printStackTrace();
	// }
	// cookies = client.getState().getCookies();
	// return cookies;
	// }

	private static Map<String, String> getCookies() throws IOException {
		File cookieFile = new File("cookies.txt");
		if (!cookieFile.exists()) {
			return null;
		}
		FileInputStream in = new FileInputStream(cookieFile);
		byte[] data = new byte[1024];

		in.read(data);
		String str = new String(data);
		String[] map = str.split(";");
		Map<String, String> cookieMap = new HashMap<String, String>();
		for (String m : map) {
			String[] c = m.split("=");
			if (c.length > 1) {
				cookieMap.put(c[0], m.substring(m.indexOf("=") + 1));
			} else {
				cookieMap.put(c[0], "");
			}
		}
		in.close();
		return cookieMap;
	}

	public static void main(String[] args) throws SAXException, XPatherException, ParserConfigurationException, XPathExpressionException {
		try {
			Map<String, String> cookies = getCookies();

			for (int i = 1; i <= 100; i++) {
				Response response = Jsoup
						.connect("https://wh.lianjia.com/ershoufang/pg" + i+"/")
						.cookies(cookies).method(Method.GET).execute();
				String body = response.body();
				
				HtmlCleaner hc = new HtmlCleaner();
				TagNode tn = hc.clean(body);
				
				Document dom = new DomSerializer(new CleanerProperties()).createDOM(tn);
				XPath xPath = XPathFactory.newInstance().newXPath();
				Object result;
				for(int k=1;k<=30;k++) {
					String exp1 = "/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[1]/a";//标题
					String exp2 = "/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[2]/div/a";//小区
					String exp3 = "/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[2]/div/text()";// | 3室2厅 | 129.52平米 | 南 | 精装
					String exp4 = "/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[3]/div/text()";
					String exp5 = "/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[6]/div[1]/span";//总价
					String exp6 = "/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[6]/div[2]/span";//单价
					
					String exp7 = "/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[1]/a";//二级页面
					
					String[] exps = {"/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[1]/a","/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[2]/div/a",
							"/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[2]/div/text()","/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[3]/div/text()",
							"/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[6]/div[1]/span","/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[6]/div[2]/span","/html/body/div[4]/div[1]/ul/li["+k+"]/div[1]/div[1]/a"};
					System.out.println("*******************************************************");
					for(int t=0;t<7;t++) {
						result = xPath.evaluate(exps[t], dom, XPathConstants.NODESET);
						
						if (result instanceof NodeList) {
							NodeList nodeList = (NodeList) result;
							//System.out.println(nodeList.getLength());
							for (int j = 0; j < nodeList.getLength(); j++) {
								Node node = nodeList.item(j);
								//System.out.println(node.getAttributes().getNamedItem("href").getNodeValue());
								System.out.println(node.getNodeValue() == null ? node.getTextContent() : node.getNodeValue());
								if(t==6) {
									Response response2 = Jsoup
											.connect(node.getAttributes().getNamedItem("href").getNodeValue())
											.cookies(cookies).method(Method.GET).execute();
									//System.out.println(node.getAttributes().getNamedItem("href").getNodeValue());
									String body2 = response2.body();
									
									HtmlCleaner hc2 = new HtmlCleaner();
									TagNode tn2 = hc2.clean(body2);
									
									Document dom2 = new DomSerializer(new CleanerProperties()).createDOM(tn2);
									Object result2;
									String[] expSub = {"/html/body/div[5]/div[2]/div[4]/div[2]/span[2]/a[1]","/html/body/div[5]/div[2]/div[4]/div[1]/a[1]","/html/body/div[5]/div[2]/div[4]/div[2]/span[2]/a[2]",
											"/html/body/div[5]/div[2]/div[4]/div[2]/span[2]/text()[2]","/html/body/div[5]/div[2]/div[4]/div[2]/a","//*[@id='introduction']/div/div/div[2]/div[2]/ul/li[6]/text()"};
									for(int u=0;u<6;u++) {
										result2 = xPath.evaluate(expSub[u], dom2, XPathConstants.NODESET);
										if(result2 instanceof NodeList) {
											NodeList nodeList2 = (NodeList) result2;
											//System.out.println(nodeList2.getLength());
											for(int jj=0;jj<nodeList2.getLength();jj++) {
												Node node2 = nodeList2.item(jj);
												System.out.println(node2.getNodeValue() == null ? node2.getTextContent() : node2.getNodeValue());
											}
										}
									}
								}
							}
						}
					}
				}

			}
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

}
